{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T07:16:38.552597Z",
     "start_time": "2020-06-15T07:16:38.512978Z"
    }
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import json\n",
    "import gc\n",
    "from numba import jit\n",
    "from tqdm import tqdm_notebook\n",
    "from tqdm import tqdm\n",
    "\n",
    "import lightgbm as lgb\n",
    "import catboost as cbt\n",
    "\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.model_selection import StratifiedKFold, KFold, RepeatedKFold\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.metrics import mean_absolute_error\n",
    "from sklearn.linear_model import LinearRegression\n",
    "from sklearn.preprocessing import StandardScaler as std\n",
    "from sklearn.kernel_ridge import KernelRidge\n",
    "from sklearn.metrics import f1_score\n",
    "import time\n",
    "import datetime\n",
    "from datetime import datetime, timedelta\n",
    "\n",
    "import gc\n",
    "from scipy.signal import hilbert\n",
    "from scipy.signal import hann\n",
    "from scipy.signal import convolve\n",
    "from scipy import stats\n",
    "import scipy.spatial.distance as dist\n",
    "from collections import Counter\n",
    "from statistics import mode\n",
    "\n",
    "import warnings\n",
    "warnings.filterwarnings(\"ignore\")\n",
    "import json\n",
    "import math\n",
    "from itertools import product\n",
    "import ast\n",
    "import re"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 科大讯飞移动广告反欺诈竞赛"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "本次比赛为参赛选手提供了5类数据：基本数据、媒体信息、时间、IP信息和设备信息。基本数据提供了广告请求会话sid，以及“是否作弊”的标识。媒体信息、时间、IP信息和设备信息等4类数据，提供了对作弊预估可能有帮助的辅助信息"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:05:33.152538Z",
     "start_time": "2020-06-15T09:05:21.799423Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 3023389 entries, 0 to 3023388\n",
      "Data columns (total 29 columns):\n",
      " #   Column        Dtype  \n",
      "---  ------        -----  \n",
      " 0   sid           object \n",
      " 1   label         int64  \n",
      " 2   pkgname       object \n",
      " 3   ver           object \n",
      " 4   adunitshowid  object \n",
      " 5   mediashowid   object \n",
      " 6   apptype       float64\n",
      " 7   nginxtime     float64\n",
      " 8   ip            object \n",
      " 9   city          object \n",
      " 10  province      float64\n",
      " 11  reqrealip     object \n",
      " 12  adidmd5       object \n",
      " 13  imeimd5       object \n",
      " 14  idfamd5       object \n",
      " 15  openudidmd5   object \n",
      " 16  macmd5        object \n",
      " 17  dvctype       float64\n",
      " 18  model         object \n",
      " 19  make          object \n",
      " 20  ntt           float64\n",
      " 21  carrier       float64\n",
      " 22  os            object \n",
      " 23  osv           object \n",
      " 24  orientation   float64\n",
      " 25  lan           object \n",
      " 26  h             float64\n",
      " 27  w             float64\n",
      " 28  ppi           float64\n",
      "dtypes: float64(10), int64(1), object(18)\n",
      "memory usage: 668.9+ MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "############# 训练数据 ####################\n",
    "\n",
    "path = '../datasets/'\n",
    "\n",
    "## 数据合并，一起做特征工程\n",
    "train1 = pd.read_table(path + \"round1_iflyad_anticheat_traindata.txt\")\n",
    "train2 = pd.read_table(path + \"round2_iflyad_anticheat_traindata.txt\")\n",
    "\n",
    "# reset_index重建索引，drop=True 是把原来索引删掉\n",
    "train = train1.append(train2).reset_index(drop=True)\n",
    "#train = pd.concat([train1,train2])\n",
    "\n",
    "print(train.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:07:35.744617Z",
     "start_time": "2020-06-15T09:07:28.424889Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 2000000 entries, 0 to 1999999\n",
      "Data columns (total 29 columns):\n",
      " #   Column        Dtype  \n",
      "---  ------        -----  \n",
      " 0   sid           object \n",
      " 1   pkgname       object \n",
      " 2   ver           object \n",
      " 3   adunitshowid  object \n",
      " 4   mediashowid   object \n",
      " 5   apptype       float64\n",
      " 6   nginxtime     float64\n",
      " 7   ip            object \n",
      " 8   city          object \n",
      " 9   province      float64\n",
      " 10  reqrealip     object \n",
      " 11  adidmd5       object \n",
      " 12  imeimd5       object \n",
      " 13  idfamd5       object \n",
      " 14  openudidmd5   object \n",
      " 15  macmd5        object \n",
      " 16  dvctype       float64\n",
      " 17  model         object \n",
      " 18  make          object \n",
      " 19  ntt           float64\n",
      " 20  carrier       float64\n",
      " 21  os            object \n",
      " 22  osv           object \n",
      " 23  orientation   float64\n",
      " 24  lan           object \n",
      " 25  h             float64\n",
      " 26  w             float64\n",
      " 27  ppi           float64\n",
      " 28  label         int64  \n",
      "dtypes: float64(10), int64(1), object(18)\n",
      "memory usage: 442.5+ MB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "############# 测试数据 #############################\n",
    "test_2a = pd.read_table(path +\n",
    "                        \"round2_iflyad_anticheat_testdata_feature_A.txt\")\n",
    "test_2b = pd.read_table(path +\n",
    "                        \"round2_iflyad_anticheat_testdata_feature_B.txt\")\n",
    "# reset_index重建索引，drop=True 是把原来索引删掉\n",
    "test = test_2a.append(test_2b).reset_index(drop=True)\n",
    "#test = pd.concat([test_2a,test_2b])\n",
    "\n",
    "test['label'] = -999  # 这样的方式是填充test 的label 列，使得test和train对其\n",
    "\n",
    "print(test.info())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:08:36.824552Z",
     "start_time": "2020-06-15T09:08:33.508841Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 5023389 entries, 0 to 5023388\n",
      "Data columns (total 29 columns):\n",
      " #   Column        Dtype  \n",
      "---  ------        -----  \n",
      " 0   sid           object \n",
      " 1   label         int64  \n",
      " 2   pkgname       object \n",
      " 3   ver           object \n",
      " 4   adunitshowid  object \n",
      " 5   mediashowid   object \n",
      " 6   apptype       float64\n",
      " 7   nginxtime     float64\n",
      " 8   ip            object \n",
      " 9   city          object \n",
      " 10  province      float64\n",
      " 11  reqrealip     object \n",
      " 12  adidmd5       object \n",
      " 13  imeimd5       object \n",
      " 14  idfamd5       object \n",
      " 15  openudidmd5   object \n",
      " 16  macmd5        object \n",
      " 17  dvctype       float64\n",
      " 18  model         object \n",
      " 19  make          object \n",
      " 20  ntt           float64\n",
      " 21  carrier       float64\n",
      " 22  os            object \n",
      " 23  osv           object \n",
      " 24  orientation   float64\n",
      " 25  lan           object \n",
      " 26  h             float64\n",
      " 27  w             float64\n",
      " 28  ppi           float64\n",
      "dtypes: float64(10), int64(1), object(18)\n",
      "memory usage: 1.1+ GB\n",
      "None\n"
     ]
    }
   ],
   "source": [
    "# reset_index重建索引，drop=True 是把原来索引删掉\n",
    "data = train.append(test).reset_index(drop=True)\n",
    "print(data.info())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 特征工程"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:09:16.228941Z",
     "start_time": "2020-06-15T09:09:16.208078Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>sid</th>\n",
       "      <th>label</th>\n",
       "      <th>pkgname</th>\n",
       "      <th>ver</th>\n",
       "      <th>adunitshowid</th>\n",
       "      <th>mediashowid</th>\n",
       "      <th>apptype</th>\n",
       "      <th>nginxtime</th>\n",
       "      <th>ip</th>\n",
       "      <th>city</th>\n",
       "      <th>...</th>\n",
       "      <th>make</th>\n",
       "      <th>ntt</th>\n",
       "      <th>carrier</th>\n",
       "      <th>os</th>\n",
       "      <th>osv</th>\n",
       "      <th>orientation</th>\n",
       "      <th>lan</th>\n",
       "      <th>h</th>\n",
       "      <th>w</th>\n",
       "      <th>ppi</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>d7460126-e071-4979-9ee8-42f72777a28a-156009070...</td>\n",
       "      <td>1</td>\n",
       "      <td>2d2664e827bcbb8b07100c7fbe072e9b</td>\n",
       "      <td>11.11.0</td>\n",
       "      <td>907d0f8c29663840491577a21c7b612a</td>\n",
       "      <td>ca64a500000d84c8fcb8a0587d0e1e0c</td>\n",
       "      <td>280.0</td>\n",
       "      <td>1.560091e+12</td>\n",
       "      <td>183.197.47.83</td>\n",
       "      <td>石家庄市</td>\n",
       "      <td>...</td>\n",
       "      <td>vivo</td>\n",
       "      <td>2.0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>Android</td>\n",
       "      <td>8.1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>zh-CN</td>\n",
       "      <td>2340.0</td>\n",
       "      <td>1080.0</td>\n",
       "      <td>3.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>b660d559-db97-4b5f-9bd2-2450cb89ce77-156005074...</td>\n",
       "      <td>1</td>\n",
       "      <td>empty</td>\n",
       "      <td>NaN</td>\n",
       "      <td>10199dc8fea2e173525bc3151bd84312</td>\n",
       "      <td>3f2fc57a6e1f9c6fa4464c25cc1e88a3</td>\n",
       "      <td>319.0</td>\n",
       "      <td>1.560051e+12</td>\n",
       "      <td>106.34.14.149</td>\n",
       "      <td>开封市</td>\n",
       "      <td>...</td>\n",
       "      <td>HUAWEI</td>\n",
       "      <td>5.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>android</td>\n",
       "      <td>Android_9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1080.0</td>\n",
       "      <td>2040.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f49a740e-66c3-4605-9b67-4d3079fe69cb-156008914...</td>\n",
       "      <td>0</td>\n",
       "      <td>16b81f93f4b1a35cebbf15f07683f171</td>\n",
       "      <td>3.2.1.0524.1958</td>\n",
       "      <td>83f2ecfe65f936f5f2ed59f8e8ff1d01</td>\n",
       "      <td>eea7280e1a2313e4c2e89290b01d196c</td>\n",
       "      <td>273.0</td>\n",
       "      <td>1.560089e+12</td>\n",
       "      <td>223.104.16.151</td>\n",
       "      <td>长春市</td>\n",
       "      <td>...</td>\n",
       "      <td>OPPO</td>\n",
       "      <td>2.0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>Android</td>\n",
       "      <td>8.1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>zh-CN</td>\n",
       "      <td>2196.0</td>\n",
       "      <td>1080.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>fd60d096-f168-4540-b782-729d64d0fcc6-156006253...</td>\n",
       "      <td>0</td>\n",
       "      <td>empty</td>\n",
       "      <td>4.7.5</td>\n",
       "      <td>9f1eadd9092b19bc86ee0cacde1c867f</td>\n",
       "      <td>eec946a5a66c023ec9d3b2ede5900626</td>\n",
       "      <td>265.0</td>\n",
       "      <td>1.560063e+12</td>\n",
       "      <td>223.104.239.101</td>\n",
       "      <td>曲靖市</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>6.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>android</td>\n",
       "      <td>7.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>a037b032-a5c7-40ea-9161-26b118b12406-156007938...</td>\n",
       "      <td>1</td>\n",
       "      <td>cf4821986014a7fef9d7b7ad8de655e4</td>\n",
       "      <td>228</td>\n",
       "      <td>2af944462e43cd2f59acbbfd37445413</td>\n",
       "      <td>57b3053174973702549ba88b6017ac30</td>\n",
       "      <td>336.0</td>\n",
       "      <td>1.560079e+12</td>\n",
       "      <td>220.152.155.170</td>\n",
       "      <td>深圳市</td>\n",
       "      <td>...</td>\n",
       "      <td>EML-AL00</td>\n",
       "      <td>2.0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>Android</td>\n",
       "      <td>9</td>\n",
       "      <td>0.0</td>\n",
       "      <td>Zh-CN</td>\n",
       "      <td>2244.0</td>\n",
       "      <td>1080.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>7ac81a32-eefe-4fe9-9429-0ecb7be5d6b9-156006758...</td>\n",
       "      <td>1</td>\n",
       "      <td>170a88a12e36f8a0f1b73442304398b1</td>\n",
       "      <td>30928000</td>\n",
       "      <td>f67a95c5c748a5bc252d3e854f8e4977</td>\n",
       "      <td>d53d2af198ebef9544f0823c3c8e84f8</td>\n",
       "      <td>301.0</td>\n",
       "      <td>1.560068e+12</td>\n",
       "      <td>223.81.196.94</td>\n",
       "      <td>潍坊市</td>\n",
       "      <td>...</td>\n",
       "      <td>vivo</td>\n",
       "      <td>2.0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>Android</td>\n",
       "      <td>8.1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>zh-CN</td>\n",
       "      <td>760.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>18deaade-64b3-4cc1-9fc8-e847f744d973-156004943...</td>\n",
       "      <td>0</td>\n",
       "      <td>b15030107319685361f4c8c0d9598cf1</td>\n",
       "      <td>NaN</td>\n",
       "      <td>4f2f5735b281a22587f22fb0aaba707e</td>\n",
       "      <td>651673a5531dde9cd40f08d09d16fe70</td>\n",
       "      <td>375.0</td>\n",
       "      <td>1.560049e+12</td>\n",
       "      <td>117.175.96.33</td>\n",
       "      <td>宜宾市</td>\n",
       "      <td>...</td>\n",
       "      <td>OPPO</td>\n",
       "      <td>2.0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>android</td>\n",
       "      <td>6.0.1</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>1280.0</td>\n",
       "      <td>720.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>a2d14df0-d499-4856-a330-8595f2923e21-156003109...</td>\n",
       "      <td>1</td>\n",
       "      <td>170a88a12e36f8a0f1b73442304398b1</td>\n",
       "      <td>30902000</td>\n",
       "      <td>4404268364e58fbf10c965a3bbc91924</td>\n",
       "      <td>d53d2af198ebef9544f0823c3c8e84f8</td>\n",
       "      <td>301.0</td>\n",
       "      <td>1.560031e+12</td>\n",
       "      <td>27.188.221.141</td>\n",
       "      <td>邯郸市</td>\n",
       "      <td>...</td>\n",
       "      <td>HUAWEI</td>\n",
       "      <td>2.0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>Android</td>\n",
       "      <td>9.0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>zh-CN</td>\n",
       "      <td>747.0</td>\n",
       "      <td>360.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>c0f81859-8256-4474-bb32-2f3ed3700e19-156006715...</td>\n",
       "      <td>0</td>\n",
       "      <td>9fa14a798a23957226e066966594d7da</td>\n",
       "      <td>NaN</td>\n",
       "      <td>d0267547ad324d53078c09095c2e1148</td>\n",
       "      <td>06765782b6564446531bf6795f0fdc48</td>\n",
       "      <td>336.0</td>\n",
       "      <td>1.560067e+12</td>\n",
       "      <td>223.104.96.81</td>\n",
       "      <td>贵阳市</td>\n",
       "      <td>...</td>\n",
       "      <td>vivo</td>\n",
       "      <td>6.0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>android</td>\n",
       "      <td>8.1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>db524c74-d9f7-41ed-bf2e-f38cdea4e15b-156005771...</td>\n",
       "      <td>1</td>\n",
       "      <td>3068122d33c1b72edb1f0d644f27405c</td>\n",
       "      <td>8.21.2</td>\n",
       "      <td>1ccb9164954c65e5e2fb5c7e176adb36</td>\n",
       "      <td>8832ab73ad09162a017fda9219ac2778</td>\n",
       "      <td>207.0</td>\n",
       "      <td>1.560058e+12</td>\n",
       "      <td>117.182.16.211</td>\n",
       "      <td>来宾市</td>\n",
       "      <td>...</td>\n",
       "      <td>OPPO</td>\n",
       "      <td>3.0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>android</td>\n",
       "      <td>8.1.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>zh-CN</td>\n",
       "      <td>1520.0</td>\n",
       "      <td>720.0</td>\n",
       "      <td>320.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 29 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                 sid  label  \\\n",
       "0  d7460126-e071-4979-9ee8-42f72777a28a-156009070...      1   \n",
       "1  b660d559-db97-4b5f-9bd2-2450cb89ce77-156005074...      1   \n",
       "2  f49a740e-66c3-4605-9b67-4d3079fe69cb-156008914...      0   \n",
       "3  fd60d096-f168-4540-b782-729d64d0fcc6-156006253...      0   \n",
       "4  a037b032-a5c7-40ea-9161-26b118b12406-156007938...      1   \n",
       "5  7ac81a32-eefe-4fe9-9429-0ecb7be5d6b9-156006758...      1   \n",
       "6  18deaade-64b3-4cc1-9fc8-e847f744d973-156004943...      0   \n",
       "7  a2d14df0-d499-4856-a330-8595f2923e21-156003109...      1   \n",
       "8  c0f81859-8256-4474-bb32-2f3ed3700e19-156006715...      0   \n",
       "9  db524c74-d9f7-41ed-bf2e-f38cdea4e15b-156005771...      1   \n",
       "\n",
       "                            pkgname              ver  \\\n",
       "0  2d2664e827bcbb8b07100c7fbe072e9b          11.11.0   \n",
       "1                             empty              NaN   \n",
       "2  16b81f93f4b1a35cebbf15f07683f171  3.2.1.0524.1958   \n",
       "3                             empty            4.7.5   \n",
       "4  cf4821986014a7fef9d7b7ad8de655e4              228   \n",
       "5  170a88a12e36f8a0f1b73442304398b1         30928000   \n",
       "6  b15030107319685361f4c8c0d9598cf1              NaN   \n",
       "7  170a88a12e36f8a0f1b73442304398b1         30902000   \n",
       "8  9fa14a798a23957226e066966594d7da              NaN   \n",
       "9  3068122d33c1b72edb1f0d644f27405c           8.21.2   \n",
       "\n",
       "                       adunitshowid                       mediashowid  \\\n",
       "0  907d0f8c29663840491577a21c7b612a  ca64a500000d84c8fcb8a0587d0e1e0c   \n",
       "1  10199dc8fea2e173525bc3151bd84312  3f2fc57a6e1f9c6fa4464c25cc1e88a3   \n",
       "2  83f2ecfe65f936f5f2ed59f8e8ff1d01  eea7280e1a2313e4c2e89290b01d196c   \n",
       "3  9f1eadd9092b19bc86ee0cacde1c867f  eec946a5a66c023ec9d3b2ede5900626   \n",
       "4  2af944462e43cd2f59acbbfd37445413  57b3053174973702549ba88b6017ac30   \n",
       "5  f67a95c5c748a5bc252d3e854f8e4977  d53d2af198ebef9544f0823c3c8e84f8   \n",
       "6  4f2f5735b281a22587f22fb0aaba707e  651673a5531dde9cd40f08d09d16fe70   \n",
       "7  4404268364e58fbf10c965a3bbc91924  d53d2af198ebef9544f0823c3c8e84f8   \n",
       "8  d0267547ad324d53078c09095c2e1148  06765782b6564446531bf6795f0fdc48   \n",
       "9  1ccb9164954c65e5e2fb5c7e176adb36  8832ab73ad09162a017fda9219ac2778   \n",
       "\n",
       "   apptype     nginxtime               ip  city  ...      make  ntt  carrier  \\\n",
       "0    280.0  1.560091e+12    183.197.47.83  石家庄市  ...      vivo  2.0  46000.0   \n",
       "1    319.0  1.560051e+12    106.34.14.149   开封市  ...    HUAWEI  5.0      0.0   \n",
       "2    273.0  1.560089e+12   223.104.16.151   长春市  ...      OPPO  2.0  46000.0   \n",
       "3    265.0  1.560063e+12  223.104.239.101   曲靖市  ...       NaN  6.0      0.0   \n",
       "4    336.0  1.560079e+12  220.152.155.170   深圳市  ...  EML-AL00  2.0  46000.0   \n",
       "5    301.0  1.560068e+12    223.81.196.94   潍坊市  ...      vivo  2.0  46000.0   \n",
       "6    375.0  1.560049e+12    117.175.96.33   宜宾市  ...      OPPO  2.0  46000.0   \n",
       "7    301.0  1.560031e+12   27.188.221.141   邯郸市  ...    HUAWEI  2.0  46000.0   \n",
       "8    336.0  1.560067e+12    223.104.96.81   贵阳市  ...      vivo  6.0  46000.0   \n",
       "9    207.0  1.560058e+12   117.182.16.211   来宾市  ...      OPPO  3.0  46000.0   \n",
       "\n",
       "        os        osv orientation    lan       h       w    ppi  \n",
       "0  Android      8.1.0         0.0  zh-CN  2340.0  1080.0    3.0  \n",
       "1  android  Android_9         0.0    NaN  1080.0  2040.0    0.0  \n",
       "2  Android      8.1.0         1.0  zh-CN  2196.0  1080.0    0.0  \n",
       "3  android        7.0         0.0    NaN     0.0     0.0    0.0  \n",
       "4  Android          9         0.0  Zh-CN  2244.0  1080.0    0.0  \n",
       "5  Android      8.1.0         1.0  zh-CN   760.0   360.0    0.0  \n",
       "6  android      6.0.1         0.0    NaN  1280.0   720.0    0.0  \n",
       "7  Android      9.0.0         1.0  zh-CN   747.0   360.0    0.0  \n",
       "8  android      8.1.0         0.0    NaN     0.0     0.0    0.0  \n",
       "9  android      8.1.0         1.0  zh-CN  1520.0   720.0  320.0  \n",
       "\n",
       "[10 rows x 29 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:14:17.170446Z",
     "start_time": "2020-06-15T09:12:44.699615Z"
    }
   },
   "outputs": [],
   "source": [
    "######## 1. 制造商预处理 ################\n",
    "data['make']\n",
    "\n",
    "#### 归类，同样的制造商可能有不同的值 ####\n",
    "\n",
    "data.loc[data['make'].str.contains('长虹', na=False), 'make'] = 'changhong'\n",
    "data.loc[data['make'].str.contains('朵唯', na=False), 'make'] = 'doov'\n",
    "data.loc[data['make'].str.contains('sm', na=False), 'make'] = 'samsung'\n",
    "data.loc[data['make'].str.contains('SAMSUNG', na=False), 'make'] = 'samsung'\n",
    "data.loc[data['make'].str.contains('三星', na=False), 'make'] = 'samsung'\n",
    "data.loc[data['make'].str.contains('GT-', na=False), 'make'] = 'samsung'\n",
    "data.loc[data['make'].str.contains('格力', na=False), 'make'] = 'gree'\n",
    "data.loc[data['make'].str.contains('Moto G', na=False), 'make'] = 'motorola'\n",
    "data.loc[data['make'].str.contains('Moto', na=False), 'make'] = 'motorola'\n",
    "data.loc[data['make'].str.contains('moto', na=False), 'make'] = 'motorola'\n",
    "data.loc[data['make'].str.contains('摩托罗拉', na=False), 'make'] = 'motorola'\n",
    "data.loc[data['make'].str.contains('诺基亚', na=False), 'make'] = 'nokia'\n",
    "data.loc[data['make'].str.contains('Nokia', na=False), 'make'] = 'nokia'\n",
    "data.loc[data['make'].str.contains('努比亚', na=False), 'make'] = 'nubia'\n",
    "data.loc[data['make'].str.contains('美图', na=False), 'make'] = 'meitu'\n",
    "data.loc[data['make'].str.contains('LG-', na=False), 'make'] = 'LG'\n",
    "data.loc[data['make'].str.contains('联想', na=False), 'make'] = 'lenovo'\n",
    "data.loc[data['make'].str.contains('rv:', na=False), 'make'] = 'RV'\n",
    "data.loc[data['make'].str.contains('rv:', na=False), 'make'] = 'RV'\n",
    "data.loc[data['make'].str.contains('小辣椒', na=False), 'make'] = 'xiaolajiao'\n",
    "data.loc[data['make'].str.contains('HUAWEI', na=False), 'make'] = 'huawei'\n",
    "data.loc[data['make'].str.contains('huawei', na=False), 'make'] = 'huawei'\n",
    "data.loc[data['make'].str.contains('荣耀', na=False), 'make'] = 'huawei'\n",
    "data.loc[data['make'].str.contains('华为', na=False), 'make'] = 'huawei'\n",
    "data.loc[data['make'].str.contains('-L', na=False), 'make'] = 'huawei'\n",
    "data.loc[data['make'].str.contains('al', na=False), 'make'] = 'huawei'\n",
    "data.loc[data['make'].str.contains('Blade', na=False), 'make'] = 'zte'\n",
    "data.loc[data['make'].str.contains('BLADE', na=False), 'make'] = 'zte'\n",
    "data.loc[data['make'].str.contains('中兴', na=False), 'make'] = 'zte'\n",
    "data.loc[data['make'].str.contains('海信', na=False), 'make'] = 'hisense'\n",
    "data.loc[data['make'].str.contains('Linux', na=False), 'make'] = 'Linux'\n",
    "data.loc[data['make'].str.contains('乐丰', na=False), 'make'] = 'lephone'\n",
    "data.loc[data['make'].str.contains('百立丰', na=False), 'make'] = 'lephone'\n",
    "data.loc[data['make'].str.contains('乐视', na=False), 'make'] = 'letv'\n",
    "data.loc[data['make'].str.contains('XT', na=False), 'make'] = 'Sony'\n",
    "data.loc[data['make'].str.contains('htc', na=False), 'make'] = 'htc'\n",
    "data.loc[data['make'].str.contains('HTC', na=False), 'make'] = 'htc'\n",
    "data.loc[data['make'].str.contains('ASUS', na=False), 'make'] = 'Asus'\n",
    "data.loc[data['make'].str.contains('锤子', na=False), 'make'] = 'smartisan'\n",
    "data.loc[data['make'].str.contains('oppo', na=False), 'make'] = 'oppo'\n",
    "data.loc[data['make'].str.contains('pb', na=False), 'make'] = 'oppo'\n",
    "data.loc[data['make'].str.contains('realme', na=False), 'make'] = 'oppo'\n",
    "data.loc[data['make'].str.contains('天语', na=False), 'make'] = 'k touch'\n",
    "data.loc[data['make'].str.contains('Tianyu', na=False), 'make'] = 'k touch'\n",
    "data.loc[data['make'].str.contains('tianyu', na=False), 'make'] = 'k touch'\n",
    "data.loc[data['make'].str.contains('酷派', na=False), 'make'] = 'coolpad'\n",
    "data.loc[data['make'].str.contains('索尼', na=False), 'make'] = 'sony'\n",
    "data.loc[data['make'].str.contains('SONY', na=False), 'make'] = 'sony'\n",
    "data.loc[data['make'].str.contains('sony', na=False), 'make'] = 'sony'\n",
    "data.loc[data['make'].str.contains('小米', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('mi', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('m1', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('m1s', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('m2', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('m2s', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('m2a', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('m3', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('m6', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('xiaomi', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('redmi', na=False), 'make'] = 'xiaomi'\n",
    "data.loc[data['make'].str.contains('魅族', na=False), 'make'] = 'meizu'\n",
    "data.loc[data['make'].str.contains('360', na=False), 'make'] = '360'\n",
    "data.loc[data['make'].str.contains('三星', na=False), 'make'] = 'samsung'\n",
    "data.loc[data['make'].str.contains('赛博宇华', na=False), 'make'] = 'sop'\n",
    "data.loc[data['make'].str.contains('金立', na=False), 'make'] = 'jinli'\n",
    "data.loc[data['make'].str.contains('gionee', na=False), 'make'] = 'jinli'\n",
    "data.loc[data['make'].str.contains('Gionee', na=False), 'make'] = 'jinli'\n",
    "data.loc[data['make'].str.contains('vivo', na=False), 'make'] = 'vivo'\n",
    "data.loc[data['make'].str.contains('VIVO', na=False), 'make'] = 'vivo'\n",
    "data.loc[data['make'].str.contains('oneplus', na=False), 'make'] = 'oneplus'\n",
    "data.loc[data['make'].str.contains('一加', na=False), 'make'] = 'oneplus'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:14:33.022293Z",
     "start_time": "2020-06-15T09:14:32.528699Z"
    }
   },
   "outputs": [],
   "source": [
    "######## 2. 型号预处理 ################\n",
    "\n",
    "# 将一些无处理的手机型号替代成能识别的手机信号\n",
    "data['model'].replace('PACM00',\"OPPO R15\",inplace=True)\n",
    "data['model'].replace('PBAM00',\"OPPO A5\",inplace=True)\n",
    "data['model'].replace('PBEM00',\"OPPO R17\",inplace=True)\n",
    "data['model'].replace('PADM00',\"OPPO A3\",inplace=True)\n",
    "data['model'].replace('PBBM00',\"OPPO A7\",inplace=True)\n",
    "data['model'].replace('PAAM00',\"OPPO R15_1\",inplace=True)\n",
    "data['model'].replace('PACT00',\"OPPO R15_2\",inplace=True)\n",
    "data['model'].replace('PABT00',\"OPPO A5_1\",inplace=True)\n",
    "data['model'].replace('PBCM10',\"OPPO R15x\",inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:32:06.239563Z",
     "start_time": "2020-06-15T09:31:46.071558Z"
    }
   },
   "outputs": [],
   "source": [
    "######## 3. 时间处理 #################\n",
    "\n",
    "## nginxtime\tbigint\t请求到达服务时间，单位 ms\n",
    "# 通过转化成时间戳转化为天和小时的特征\n",
    "# 时间特征\n",
    "data['time'] = pd.to_datetime(data['nginxtime'] / 1000,\n",
    "                              unit='s') + timedelta(hours=8)\n",
    "data['day'] = data['time'].dt.day\n",
    "data['hour'] = data['time'].dt.hour\n",
    "\n",
    "## 请求sid 里面有一个 到达的时间搓将时间搓切割出来\n",
    "# sid 时间戳\n",
    "data['sid_timestamp'] = data['sid'].apply(\n",
    "    lambda x: str(x).split('-')[-1]).astype(str)\n",
    "\n",
    "## 请求时间和到达时间相差\n",
    "## 判断请求时间和到达时间是否相等\n",
    "data['nginxtime_diff_sid_time'] = data['nginxtime'].astype(\n",
    "    'float') - data['sid_timestamp'].astype('float')\n",
    "data['nginxtime_sid_timestamp'] = (\n",
    "    data['nginxtime'] == data['sid_timestamp'].astype(str)).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:33:43.816892Z",
     "start_time": "2020-06-15T09:33:39.429118Z"
    }
   },
   "outputs": [],
   "source": [
    "######## 4. 将操作系统标准化 ####################\n",
    "\n",
    "# 将 os 属性标准化，都转换成大写\n",
    "data['os'] = data[data.os.notnull()]['os'].apply(lambda x: str(x).upper())\n",
    "data['os'] = data['os'].map(lambda x: str(x).upper())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T09:43:59.816160Z",
     "start_time": "2020-06-15T09:43:57.385273Z"
    }
   },
   "outputs": [],
   "source": [
    "######## 5. 手机型号和手机制造商 ##################\n",
    "\n",
    "data['big_model'] = data['model'].map(lambda x: str(x).split(' ')[0])\n",
    "\n",
    "#制造商和型号是否相等\n",
    "data['model_equal_make'] = (data['big_model'] == data['make']).astype(int)\n",
    "\n",
    "# 制造商和机型进行交叉\n",
    "data['machine'] = data['make'].astype(str) + '-' + data['model'].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T10:27:36.095293Z",
     "start_time": "2020-06-15T10:27:33.362690Z"
    }
   },
   "outputs": [],
   "source": [
    "######## 6. 手机尺寸信息处理 ##################\n",
    "\n",
    "import re\n",
    "data['h'].replace(0.0, np.nan, inplace=True)\n",
    "data['w'].replace(0.0, np.nan, inplace=True)\n",
    "# all_data['ppi'].replace(0.0, np.nan, inplace=True)\n",
    "# cols = ['h', 'w', 'ppi']\n",
    "\n",
    "cols = ['h', 'w']\n",
    "gp_col = 'make'\n",
    "for col in tqdm(cols):\n",
    "    na_series = data[col].isna()\n",
    "    names = list(data.loc[na_series, gp_col])\n",
    "    \n",
    "    # 使用均值 或者众数进行填充缺失值\n",
    "    # df_fill = all_data.groupby(gp_col)[col].mean()\n",
    "    df_fill = data.groupby(gp_col)[col].agg(lambda x: stats.mode(x)[0][0])\n",
    "    \n",
    "    t = df_fill.loc[names]\n",
    "    \n",
    "    t.index = data.loc[na_series, col].index\n",
    "    # 相同的index进行赋值\n",
    "    data.loc[na_series, col] = t"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T10:30:02.895092Z",
     "start_time": "2020-06-15T10:29:51.610532Z"
    }
   },
   "outputs": [],
   "source": [
    "####### 7. ip 处理 ########################\n",
    "\n",
    "data['ip0'] = data['ip'].apply(lambda x: '.'.join(str(x).split('.')[:1]))\n",
    "data['ip1'] = data['ip'].apply(lambda x: '.'.join(str(x).split('.')[0:2]))\n",
    "data['ip2'] = data['ip'].apply(lambda x: '.'.join(str(x).split('.')[0:3]))\n",
    "\n",
    "data['reqrealip0'] = data['reqrealip'].apply(\n",
    "    lambda x: '.'.join(str(x).split('.')[:1]))\n",
    "data['reqrealip1'] = data['reqrealip'].apply(\n",
    "    lambda x: '.'.join(str(x).split('.')[0:2]))\n",
    "data['reqrealip2'] = data['reqrealip'].apply(\n",
    "    lambda x: '.'.join(str(x).split('.')[0:3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T10:30:15.810236Z",
     "start_time": "2020-06-15T10:30:15.459141Z"
    }
   },
   "outputs": [],
   "source": [
    "# 判断二者是否相等\n",
    "data['ip_reqrealip'] = (data['ip'] == data['reqrealip']).astype(int)\n",
    "data['adidmd5_openudidmd5_d'] = (\n",
    "    data['adidmd5'] == data['openudidmd5']).astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T10:30:51.300405Z",
     "start_time": "2020-06-15T10:30:46.893896Z"
    }
   },
   "outputs": [],
   "source": [
    "####### 8. 操作系统版本 ##############\n",
    "data['osv0'] = data['osv'].astype(str).map(\n",
    "    lambda x: '.'.join(x.split('.')[:1]))\n",
    "data['osv1'] = data['osv'].astype(str).map(\n",
    "    lambda x: '.'.join(x.split('.')[0:2]))\n",
    "data['osv2'] = data['osv'].astype(str).map(\n",
    "    lambda x: '.'.join(x.split('.')[0:3]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T10:34:08.614945Z",
     "start_time": "2020-06-15T10:32:31.958310Z"
    }
   },
   "outputs": [],
   "source": [
    "####### 9. 特征进行交叉 ####################\n",
    "\n",
    "#既然特征在catboost上面表现优越，在原来类别特征之间找新的交叉特征\n",
    "### ip 暴力与设备进行暴力交叉\n",
    "## 找城市，省份欺诈设备族群,ip 和 抬头ip过于稀疏，所以只拿province 和city 和设备\n",
    "###['lan','make','model','osv','dvctype','imeimd5','os']暂时就这些，其他的nunique 过于少\n",
    "#剔除重要性差特征\n",
    "province_city = ['province', 'city']\n",
    "pc_cos = ['make', 'model', 'osv', 'dvctype', 'imeimd5', 'lan', 'macmd5']  #os,\n",
    "for i in province_city:\n",
    "    for j in pc_cos:\n",
    "        data['twoPC_' + str(i) + '_' +\n",
    "             str(j)] = data[i].astype(str) + '_' + data[j].astype(str)\n",
    "        \n",
    "        \n",
    "#媒体暴力交互\n",
    "##媒体信息和设备交互欺诈族群寻找\n",
    "###媒体信息有 ['pkgname','ver','adunitshowid','mediashowid'，'apptype']\n",
    "###设备信息有['adidmd5','imeimd5','idfamd5','openudidmd5','macmd5','model','make','carrier','osv','lan']剔除了仅仅只有os\n",
    "### 将媒体信息与全部设备信息进行暴力交叉\n",
    "Media_Information = [\n",
    "    'pkgname', 'ver', 'adunitshowid', 'mediashowid', 'apptype'\n",
    "]  #按顺序开始取字段\n",
    "Device_Information = [\n",
    "    'adidmd5', 'imeimd5', 'openudidmd5', 'macmd5', 'model', 'make', 'carrier',\n",
    "    'osv', 'lan'\n",
    "]  #,'idfamd5'\n",
    "for ii in Media_Information:\n",
    "    for jj in Device_Information:\n",
    "        data['twoMD_' + str(ii) + '_' +\n",
    "             str(jj)] = data[ii].astype(str) + '_' + data[jj].astype(str)\n",
    "        \n",
    "        \n",
    "        \n",
    "#ip 和默认下载匹配app交互暴力交互ip\n",
    "Ip_Information = ['ip', 'reqrealip']\n",
    "Id_Information = ['adunitshowid']  #'mediashowid','mediashowid',好像过拟合\n",
    "for ii in Ip_Information:\n",
    "    for jj in Id_Information:\n",
    "        data['twoMD_' + str(ii) + '_' +\n",
    "             str(jj)] = data[ii].astype(str) + '_' + data[jj].astype(str)\n",
    "        \n",
    "        \n",
    "        \n",
    "## 强特'imeimd5'暴力交叉\n",
    "##设备信息自身类别交互。先选出设备唯一id，一般设备id为International Mobile Equipment Identity IMEI识别码。\n",
    "Za_Information = ['model', 'osv', 'ver', 'dvctype', 'carrier']  #按顺序开始取字段\n",
    "Imd_Information = ['imeimd5']  #,'idfamd5',其他ID好像过拟合了\n",
    "for ii in Za_Information:\n",
    "    for jj in Imd_Information:\n",
    "        data['twoMD_' + str(ii) + '_' +\n",
    "             str(jj)] = data[ii].astype(str) + '_' + data[jj].astype(str)\n",
    "\n",
    "        \n",
    "##设备信息自身类别交互。通过操作系统和本身设备交互\n",
    "### 看看ime5和其他设备和部分设备是如何交互\n",
    "Device_Information3 = ['model', 'make', 'ntt']\n",
    "for iiii in Device_Information3:\n",
    "    data['twoMD_' + str('osv') + '_' +\n",
    "         str(iiii)] = data['osv'].astype(str) + '_' + data[iiii].astype(str)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T10:34:19.302004Z",
     "start_time": "2020-06-15T10:34:19.218025Z"
    }
   },
   "outputs": [],
   "source": [
    "data['size'] = (np.sqrt(data['h']**2 + data['w']**2) / 2.54) / 1000\n",
    "data['ratio'] = data['h'] / data['w']\n",
    "data['px'] = data['ppi'] * data['size']\n",
    "data['mj'] = data['h'] * data['w']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T10:34:44.245020Z",
     "start_time": "2020-06-15T10:34:27.814916Z"
    }
   },
   "outputs": [],
   "source": [
    "###查看该用户id用过多少个机型和设备等等(人与人的交互)开源特征：\n",
    "sid_cro = [\n",
    "    'ver', 'apptype', 'ntt', 'h', 'w', 'ppi', 'ratio', 'city', 'dvctype'\n",
    "]\n",
    "for x in sid_cro:\n",
    "    data['sid' + '_' + 'use' + '_' + str(x) + '_count'] = data.groupby(\n",
    "        [x])['sid'].transform('count')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "ExecuteTime": {
     "end_time": "2020-06-15T10:49:29.838392Z",
     "start_time": "2020-06-15T10:34:49.762578Z"
    }
   },
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "9f84d5be18e24cc4bdae09726c22e6c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "HBox(children=(FloatProgress(value=0.0, max=97.0), HTML(value='')))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "############## 类别特征转换成数值特征 #######################\n",
    "\n",
    "cat_col = [\n",
    "    i for i in data.select_dtypes(object).columns if i not in ['sid', 'label']\n",
    "]\n",
    "for i in tqdm_notebook(cat_col):\n",
    "    lbl = LabelEncoder()\n",
    "    data['count_' + i] = data.groupby([i])[i].transform('count')\n",
    "    data[i] = lbl.fit_transform(data[i].astype(str))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "ExecuteTime": {
     "start_time": "2020-06-15T10:51:42.733Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(3023389, 227) (2000000, 227)\n",
      "0\n",
      "(2591476, 223)\n",
      "(2591476,)\n",
      "(431913, 223)\n",
      "(431913,)\n",
      "0:\tlearn: 0.9369822\ttest: 0.9506234\tbest: 0.9506234 (0)\ttotal: 1.25s\tremaining: 1h 43m 47s\n",
      "100:\tlearn: 0.9653390\ttest: 0.9673282\tbest: 0.9673282 (100)\ttotal: 2m 15s\tremaining: 1h 49m 50s\n",
      "200:\tlearn: 0.9675168\ttest: 0.9692015\tbest: 0.9692015 (200)\ttotal: 4m 32s\tremaining: 1h 48m 33s\n",
      "300:\tlearn: 0.9685575\ttest: 0.9697958\tbest: 0.9697958 (300)\ttotal: 6m 53s\tremaining: 1h 47m 34s\n",
      "400:\tlearn: 0.9690024\ttest: 0.9701823\tbest: 0.9701869 (399)\ttotal: 9m 10s\tremaining: 1h 45m 14s\n",
      "500:\tlearn: 0.9693755\ttest: 0.9703934\tbest: 0.9703934 (500)\ttotal: 11m 24s\tremaining: 1h 42m 29s\n",
      "600:\tlearn: 0.9696560\ttest: 0.9704395\tbest: 0.9704705 (581)\ttotal: 13m 41s\tremaining: 1h 40m 13s\n",
      "700:\tlearn: 0.9699012\ttest: 0.9705870\tbest: 0.9705907 (671)\ttotal: 15m 56s\tremaining: 1h 37m 46s\n",
      "800:\tlearn: 0.9701570\ttest: 0.9707058\tbest: 0.9707058 (800)\ttotal: 18m 14s\tremaining: 1h 35m 38s\n",
      "900:\tlearn: 0.9703205\ttest: 0.9707261\tbest: 0.9707565 (881)\ttotal: 20m 33s\tremaining: 1h 33m 32s\n",
      "1000:\tlearn: 0.9704878\ttest: 0.9708164\tbest: 0.9708206 (944)\ttotal: 22m 45s\tremaining: 1h 30m 57s\n",
      "1100:\tlearn: 0.9706236\ttest: 0.9708644\tbest: 0.9708844 (1083)\ttotal: 24m 59s\tremaining: 1h 28m 31s\n",
      "1200:\tlearn: 0.9707656\ttest: 0.9709622\tbest: 0.9709756 (1178)\ttotal: 27m 16s\tremaining: 1h 26m 17s\n",
      "1300:\tlearn: 0.9709234\ttest: 0.9710137\tbest: 0.9710392 (1258)\ttotal: 29m 33s\tremaining: 1h 24m 2s\n",
      "1400:\tlearn: 0.9710569\ttest: 0.9710747\tbest: 0.9710899 (1395)\ttotal: 31m 50s\tremaining: 1h 21m 48s\n",
      "1500:\tlearn: 0.9712040\ttest: 0.9711581\tbest: 0.9711581 (1500)\ttotal: 34m 9s\tremaining: 1h 19m 36s\n",
      "1600:\tlearn: 0.9713449\ttest: 0.9711835\tbest: 0.9712025 (1593)\ttotal: 36m 23s\tremaining: 1h 17m 15s\n",
      "1700:\tlearn: 0.9714987\ttest: 0.9712152\tbest: 0.9712345 (1680)\ttotal: 38m 42s\tremaining: 1h 15m 4s\n",
      "1800:\tlearn: 0.9715912\ttest: 0.9712228\tbest: 0.9712345 (1680)\ttotal: 40m 59s\tremaining: 1h 12m 48s\n",
      "1900:\tlearn: 0.9717185\ttest: 0.9712733\tbest: 0.9712820 (1886)\ttotal: 43m 13s\tremaining: 1h 10m 28s\n",
      "2000:\tlearn: 0.9718120\ttest: 0.9712207\tbest: 0.9712904 (1911)\ttotal: 45m 29s\tremaining: 1h 8m 11s\n",
      "2100:\tlearn: 0.9719391\ttest: 0.9712873\tbest: 0.9712904 (1911)\ttotal: 47m 46s\tremaining: 1h 5m 55s\n",
      "2200:\tlearn: 0.9720437\ttest: 0.9713213\tbest: 0.9713213 (2200)\ttotal: 50m 1s\tremaining: 1h 3m 36s\n",
      "2300:\tlearn: 0.9721409\ttest: 0.9713001\tbest: 0.9713300 (2207)\ttotal: 52m 20s\tremaining: 1h 1m 23s\n",
      "2400:\tlearn: 0.9722510\ttest: 0.9713091\tbest: 0.9713300 (2207)\ttotal: 54m 36s\tremaining: 59m 6s\n",
      "2500:\tlearn: 0.9723530\ttest: 0.9713185\tbest: 0.9713384 (2411)\ttotal: 56m 53s\tremaining: 56m 50s\n",
      "2600:\tlearn: 0.9724428\ttest: 0.9714014\tbest: 0.9714014 (2600)\ttotal: 59m 11s\tremaining: 54m 35s\n",
      "2700:\tlearn: 0.9725478\ttest: 0.9714579\tbest: 0.9714579 (2700)\ttotal: 1h 1m 28s\tremaining: 52m 19s\n",
      "2800:\tlearn: 0.9726498\ttest: 0.9714058\tbest: 0.9714679 (2722)\ttotal: 1h 3m 45s\tremaining: 50m 3s\n",
      "2900:\tlearn: 0.9727701\ttest: 0.9713708\tbest: 0.9714679 (2722)\ttotal: 1h 5m 59s\tremaining: 47m 45s\n",
      "3000:\tlearn: 0.9728697\ttest: 0.9714060\tbest: 0.9714679 (2722)\ttotal: 1h 8m 16s\tremaining: 45m 28s\n",
      "3100:\tlearn: 0.9730033\ttest: 0.9714226\tbest: 0.9714679 (2722)\ttotal: 1h 10m 37s\tremaining: 43m 14s\n",
      "3200:\tlearn: 0.9730842\ttest: 0.9714185\tbest: 0.9714679 (2722)\ttotal: 1h 12m 53s\tremaining: 40m 57s\n",
      "bestTest = 0.9714679129\n",
      "bestIteration = 2722\n",
      "Shrink model to first 2723 iterations.\n"
     ]
    }
   ],
   "source": [
    "############ 训练模型  ########################\n",
    "\n",
    "\n",
    "feature_name = [i for i in data.columns if i not in ['sid', 'label', 'time', 'day']]\n",
    "cat_list = [i for i in cat_col if i not in ['nginxtime']]\n",
    "\n",
    "feature_importances = pd.DataFrame()\n",
    "feature_importances['feature'] = feature_name\n",
    "all_zero_feature=[]\n",
    "X_train = data[data['label']!=-999]\n",
    "y = data[data['label']!=-999]['label']\n",
    "X_test = data[data['label']==-999]\n",
    "print(X_train.shape,X_test.shape)\n",
    "\n",
    "oof = np.zeros(X_train.shape[0])\n",
    "prediction = np.zeros(X_test.shape[0])\n",
    "seeds = [2018, 2019,4096, 2048, 1024]\n",
    "num_model_seed = 1\n",
    "\n",
    "for model_seed in range(num_model_seed):\n",
    "    oof_cat = np.zeros(X_train.shape[0])\n",
    "    prediction_cat=np.zeros(X_test.shape[0])\n",
    "    skf = StratifiedKFold(n_splits=7, random_state=seeds[model_seed], shuffle=True)\n",
    "    \n",
    "    for index, (train_index, test_index) in enumerate(skf.split(X_train, y)):\n",
    "        print(index)\n",
    "        train_x, test_x, train_y, test_y = X_train[feature_name].iloc[train_index], X_train[feature_name].iloc[test_index], y.iloc[train_index], y.iloc[test_index]\n",
    "        print(train_x.shape)\n",
    "        print(train_y.shape)\n",
    "        print(test_x.shape)\n",
    "        print(test_y.shape)\n",
    "        cbt_model = cbt.CatBoostClassifier(iterations=5000,learning_rate=0.1,max_depth=7,\n",
    "                                           l2_leaf_reg=1,verbose=100,early_stopping_rounds=500,task_type='GPU',eval_metric='F1',cat_features=cat_list)\n",
    "        cbt_model.fit(train_x[feature_name], train_y,eval_set=(test_x[feature_name],test_y))            \n",
    "        \n",
    "        oof_cat[test_index] += cbt_model.predict_proba(test_x)[:,1]\n",
    "        prediction_cat += cbt_model.predict_proba(X_test[feature_name])[:,1]/7       \n",
    "        featureimportance=cbt_model.get_feature_importance(prettified=True)   \n",
    "        importance_0=list(featureimportance[featureimportance['Importances']==0]['Feature Index'])\n",
    "        all_zero_feature.extend(importance_0)\n",
    "        \n",
    "        feature_importances['fold-{}'.format(index+1)] = cbt_model.get_feature_importance()\n",
    "        featureimportance.to_csv(path+'feats{}.csv'.format(index),index=False)  \n",
    "        \n",
    "    print('F1',f1_score(y, np.round(oof_cat)))    \n",
    "    oof += oof_cat / num_model_seed\n",
    "    prediction += prediction_cat / num_model_seed\n",
    "print('score',f1_score(y, np.round(oof))) \n",
    "# write to csv\n",
    "submit = test[['sid']]\n",
    "submit['label'] =prediction\n",
    "# submit.to_csv(path+\"tmp/bs_gl1.csv\",index=False)\n",
    "\n",
    "submit['label'] = (prediction>=0.5).astype(int)\n",
    "print(submit['label'].value_counts())\n",
    "\n",
    "# submit.to_csv(path+\"tmp/dt1.csv\",index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {},
   "toc_section_display": true,
   "toc_window_display": false
  },
  "varInspector": {
   "cols": {
    "lenName": 16,
    "lenType": 16,
    "lenVar": 40
   },
   "kernels_config": {
    "python": {
     "delete_cmd_postfix": "",
     "delete_cmd_prefix": "del ",
     "library": "var_list.py",
     "varRefreshCmd": "print(var_dic_list())"
    },
    "r": {
     "delete_cmd_postfix": ") ",
     "delete_cmd_prefix": "rm(",
     "library": "var_list.r",
     "varRefreshCmd": "cat(var_dic_list()) "
    }
   },
   "types_to_exclude": [
    "module",
    "function",
    "builtin_function_or_method",
    "instance",
    "_Feature"
   ],
   "window_display": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
